"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 硬件故障类--disk
Case Name   : 数据库减容/扩容时备节点IOwait高
Create At   : 2021/07/12
@zou_jialiang0505328126
Description :
    1.备节点（非目标节点）注入ioWait高故障
    2.1主2备，减1备  期望:减容成功
    3.检查集群中节点个数
    4.清除故障
    5.执行TPCC，并检查数据一致性
    6.备节点（非目标节点）注入ioWait高故障
    7.将减容节点还原
    8.检查集群中节点个数
    9清除故障
    10.执行TPCC，并检查数据一致性
Expect      :
    1.成功
    2.成功
    3.节点数变为2
    4.成功
    5.执行成功，主备一致
    6.成功
    7.成功
    8.节点数变为3
    9.成功
    10.执行成功，主备一致
History     :
    Modified by @wan005 2021/8/3:扩容后需手动修改synchronous_standby_names
    Modified 2021/9/1 @wan005:适配最新CFE故障打印
    Modified by wx1115623 2023/2/16:扩容前重建root互信
"""

import unittest
import time
import os
from yat.test import Node, macro
from testcase.utils.Common import Common
from testcase.utils.Constant import Constant
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Logger import Logger


class DiskIOwait(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(f"-----{os.path.basename(__file__)} start-----")
        self.primary_root_node = Node(node='PrimaryRoot')
        self.sta1_root_node = Node(node='Standby1Root')
        self.primary_user_node = Node(node='PrimaryDbUser')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.primary_sh = CommonSH('PrimaryDbUser')
        self.priroot_sh = CommonSH('PrimaryRoot')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.sta2root_sh = CommonSH('Standby2Root')
        self.host_tuple = (self.primary_user_node.ssh_host,
                           self.sta1_dbuser.ssh_host,
                           self.sta2_dbuser.ssh_host)
        self.params = {'-f': 'test_hosts'}

        cmd = f"df -h {macro.DB_INSTANCE_PATH}"
        self.log.info(cmd)
        msg = self.sta1_root_node.sh(cmd).result()
        self.log.info(msg)
        self.dir = msg.splitlines()[-1].split()[-1].strip()

        result = self.primary_sh.executDbSql(
            'show synchronous_standby_names;')
        self.log.info(f"synchronous_standby_names is {result}")
        self.synchronous_standby_names_p = result.strip().splitlines()[-2]
        result = self.sta_1_sh.executDbSql(
            'show synchronous_standby_names;')
        self.log.info(f"synchronous_standby_names is {result}")
        self.synchronous_standby_names_s1 = result.strip().splitlines()[-2]
        result = self.sta_2_sh.executDbSql(
            'show synchronous_standby_names;')
        self.log.info(f"synchronous_standby_names is {result}")
        self.synchronous_standby_names_s2 = result.strip().splitlines()[-2]

    def test_main(self):
        text = '----step1: 备节点（非目标节点）注入ioWait高故障 expect:成功----'
        self.log.info(text)
        inject_command = f"rIO_iowait (type,dir,bs,count,time) " \
            f"values (w_iowait,{self.dir}, 10000,10000,1000)"
        result = self.com.cfe_inject(self.sta1_root_node, inject_command)
        self.log.info(result)
        result = self.com.cfe_query(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        self.assertIn(self.constant.cfe_iowait_success_msg, result,
                      '执行失败:' + text)

        text = '----step2: 1主2备，减1备  expect:成功----'
        self.log.info(text)
        execute_cmd = f'''source {macro.DB_ENV_PATH};
                            expect <<EOF
                            set timeout 360
                            spawn gs_dropnode -U \
                            {self.primary_user_node.ssh_user} \
                            -G {self.primary_user_node.ssh_user} \
                            -h {self.sta2_dbuser.ssh_host}
                            expect "*drop the target node (yes/no)?*"
                            send "yes\\n"
                            expect eof\n''' + '''EOF'''
        self.log.info(execute_cmd)
        result = self.primary_user_node.sh(execute_cmd).result()
        self.log.info(result)
        self.assertIn("Success to drop the target nodes", result,
                      '执行失败:' + text)

        text = '----step3: 检查集群中节点个数 expect:成功----'
        self.log.info(text)
        sql = "select count(*) from pg_stat_get_wal_senders();"
        result = self.primary_sh.executDbSql(sql)
        self.log.info(result)
        self.assertEqual("1", result.splitlines()[-2].strip(), '执行失败:' + text)

        text = '----step4: 清除故障 expect:成功----'
        self.log.info(text)
        result = self.com.cfe_clean(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        result = self.com.cfe_query(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        self.assertNotIn(self.constant.cfe_iowait_success_msg, result,
                         '执行失败:' + text)

        text = '----step5: 执行TPCC，并检查数据一致性 expect:执行成功，主备一致----'
        self.log.info(text)
        result = self.com.startTPCC(self.primary_root_node, macro.TPCC_PATH)
        self.log.info(result)
        self.assertIn(self.constant.TPCC_SUCCESS_MSG, result, '执行失败:' + text)
        time.sleep(10)
        result = self.sta_1_sh.check_data_consistency()
        self.assertTrue(result, '执行失败:' + text)
        nodes_tuple = (self.primary_user_node, self.sta1_dbuser)
        flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
        self.assertTrue(flag, '执行失败:' + text)

        table_dict = self.com.format_sql_result(
            self.primary_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f'select count(*) from {name};'
            flag = self.com.check_data_sample_by_all(select_sql,
                                                     *nodes_tuple)
            self.assertTrue(flag, '执行失败:' + text)

        text = '----step6: 备节点（非目标节点）注入ioWait高故障 expect:成功----'
        self.log.info(text)
        inject_command = f"rIO_iowait (type,dir,bs,count,time) " \
            f"values (w_iowait,{self.dir}, 10000,10000,1500)"
        result = self.com.cfe_inject(self.sta1_root_node, inject_command)
        self.log.info(result)
        result = self.com.cfe_query(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        self.assertIn(self.constant.cfe_iowait_success_msg, result,
                      '执行失败:' + text)

        text = '----step7: 将减容节点还原 expect:成功----'
        self.log.info(text)
        result = self.sta2root_sh.exec_gs_sshexkey(macro.DB_SCRIPT_PATH,
                                                   *self.host_tuple,
                                                   **self.params)
        self.log.info(result)
        result = self.priroot_sh.exec_expension(
            self.primary_user_node.ssh_user, self.primary_user_node.ssh_user,
            self.sta2_dbuser.ssh_host, macro.DB_XML_PATH)
        self.assertTrue(result, '执行失败:' + text)

        text = '----step8: 检查集群中节点个数 expect:成功----'
        self.log.info(text)
        sql = "select count(*) from pg_stat_get_wal_senders();"
        result = self.primary_sh.executDbSql(sql)
        self.log.info(result)
        self.assertEqual("2", result.splitlines()[-2].strip(),
                         '执行失败:' + text)

        text = '----step9: 清除故障 expect:成功----'
        self.log.info(text)
        result = self.com.cfe_clean(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        result = self.com.cfe_query(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        self.assertNotIn(self.constant.cfe_iowait_success_msg, result,
                         '执行失败:' + text)

        text = '----step10: 执行TPCC，并检查数据一致性 expect:成功----'
        self.log.info(text)
        result = self.com.startTPCC(self.primary_root_node, macro.TPCC_PATH)
        self.log.info(result)
        self.assertIn(self.constant.TPCC_SUCCESS_MSG, result, '执行失败:' + text)
        time.sleep(10)
        result = self.sta_1_sh.check_data_consistency()
        self.assertTrue(result, '执行失败:' + text)
        result = self.sta_2_sh.check_data_consistency()
        self.assertTrue(result, '执行失败:' + text)
        nodes_tuple = (self.primary_user_node,
                       self.sta1_dbuser, self.sta2_dbuser)
        flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
        self.assertTrue(flag, '执行失败:' + text)

        table_dict = self.com.format_sql_result(
            self.primary_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f'select count(*) from {name};'
            flag = self.com.check_data_sample_by_all(select_sql,
                                                     *nodes_tuple)
            self.assertTrue(flag, '执行失败:' + text)

    def tearDown(self):
        self.log.info("--------------备节点（非目标节点）清理故障-------------")
        result = self.com.cfe_clean(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        result = self.com.cfe_query(self.sta1_root_node, "rIO_iowait")
        self.log.info(result)
        self.log.info("-------------还原集群---------------")
        sql = "select count(*) from pg_stat_get_wal_senders();"
        result = self.primary_sh.executDbSql(sql)
        self.log.info(result)
        if "2" != result.splitlines()[-2].strip():
            self.sta2root_sh.exec_gs_sshexkey(macro.DB_SCRIPT_PATH,
                                              *self.host_tuple,
                                              **self.params)
            self.priroot_sh.exec_expension(
                self.primary_user_node.ssh_user,
                self.primary_user_node.ssh_user,
                self.sta2_dbuser.ssh_host, macro.DB_XML_PATH)
        self.log.info("----------还原synchronous_standby_names-------------")
        p_hostname = self.primary_root_node.sh("hostname").result()
        self.log.info(p_hostname)
        self.primary_sh.executeGsguc('reload',
                                     self.constant.GSGUC_SUCCESS_MSG,
                                     f"synchronous_standby_names="
                                     f"'{self.synchronous_standby_names_p}'",
                                     node_name=p_hostname)
        s_hostname = self.sta1_dbuser.sh("hostname").result()
        self.log.info(s_hostname)
        self.sta_1_sh.executeGsguc('reload',
                                   self.constant.GSGUC_SUCCESS_MSG,
                                   f"synchronous_standby_names="
                                   f"'{self.synchronous_standby_names_s1}'",
                                   node_name=s_hostname)
        s_hostname = self.sta2_dbuser.sh("hostname").result()
        self.log.info(s_hostname)
        self.sta_2_sh.executeGsguc('reload',
                                   self.constant.GSGUC_SUCCESS_MSG,
                                   f"synchronous_standby_names="
                                   f"'{self.synchronous_standby_names_s2}'",
                                   node_name=s_hostname)
        self.log.info(f"-----{os.path.basename(__file__)} end-----")
